import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
}

# 获取源代码
def get_html(url):
    response = requests.get(url, headers=headers)
    result = response.content.decode('utf-8')
    soup = BeautifulSoup(result, 'lxml')

    return soup


# 获取所有详情url
def get_detail_url(url):
    html = get_html(url)
    detail_list = []
    lis = html.find('ol', attrs={'class':'grid_view'}).find_all('li')
    for li in lis:
        detail_url = li.find('a').get('href')
        detail_list.append(detail_url)

    return detail_list


# 获取详情页数据
def get_detail_data(url, f):

    html = get_html(url)
    # 电影名
    title = list(html.find('div', attrs={'id': 'content'}).find('h1').stripped_strings)

    # 导演
    director = list(
        html.find('div', attrs={'id': 'info'}).find('span').find('span', attrs={'class': 'attrs'}).stripped_strings)

    # 编剧
    screenwriter = list(html.find('div', attrs={'id': 'info'}).find_all('span')[3].find('span', attrs={
        'class': 'attrs'}).stripped_strings)

    # 主演
    starring = list(html.find('div', attrs={'id': 'info'}).find('span', attrs={'class': 'actor'}).find('span',
                                                                                                       attrs={
                                                                                                           'class': 'attrs'}).stripped_strings)

    # 评分
    num = list(html.find('strong', attrs={'class': 'll rating_num'}).stripped_strings)

    f.write('{},{},{},{},{}\n'.format(''.join(title), ''.join(director), ''.join(screenwriter), ''.join(starring),
                                    ''.join(num)))

def main():
    base_url = "https://movie.douban.com/top250?start={}&filter="
    with open('Top250.csv', 'a', encoding='utf-8') as f:
        for x in range(0,251,25):
            detail_urls = get_detail_url(base_url.format(x))
            for detail_url in detail_urls:
                get_detail_data(detail_url,f)


if __name__ == '__main__':
    main()